from cardsharp.transform import metadata
from collections import defaultdict
from datetime import datetime
from errors import *
from itertools import izip
from itertools import count as icount
from util import *
import cardsharp as cs
import codecs
import log
import re
import sre_constants
from configuration import config
__all__ = ['Rule', 'AllRule', 'RegexRule', 'Crosswalk', 'RegexMathRule']
time = datetime.now().isoformat('_').replace(':', '-')
[docs]class Rule(object):
'''Default Crosswalk Rule defining a crosswalk conversion.
When a crosswalk is loaded each line in the crosswalk file is turned
into a rule. For this default rule the initial value is the key in the
crosswalk and the result is the value in the crosswalk. See crosswalk
for more details.
Public Methods:
handle -- Returns true if initial matches the value
convert -- Returns the result
'''
def __init__(self, initial, result):
self.initial = initial
self.result = result
self.return_type = 'result'
self.type = 'match'
[docs] def convert(self, **kw):
if kw.get('split'):
return [x for x in self.result.split(kw['split'])]
else:
return self.result
[docs]class AllRule(object):
'''Crosswalk rule to handle all values.
If result is :copy: will copy the initial value to the result value,
otherwise will output the result value.
example)
:all: :copy:
This will output exactly what is in the original dataset without
doing any conversion.
:all: :none:
This will output a None value.
:all: 1111.11
This will output all values to 1111.11
'''
def __init__(self, result):
self.result = result
self.return_type = 'result'
self.copy = True if re.search(':copy:', result, re.I) else False
self.none = True if re.search(':none:', result, re.I) else False
self.type = 'all'
self.out = []
self._rules = {':copy:': lambda key: self.out.append(key[0]),
':none:': lambda key: self.out.append('')}
[docs] def convert(self, **kw):
#if multiple out value are defined
if kw.get('split'):
if self.copy or self.none:
#add the result value to out list based on its rule (:copy:, :none: or the result value)
for x in self.result.split(kw['split']):
self._rules.get(x, lambda r: self.out.append(r[1]))((kw.get('value'), x))
return self.out
#otherwise return result values as a list
else:
return [x for x in self.result.split(kw['split'])]
else:
if self.copy or self.none:
self._rules.get(self.result)((kw.get('value'), self.result))
return self.out[0]
return self.result
[docs]class RegexRule(object):
def __init__(self, regex, result, type, priority, escape=True, **kw):
"""
@param regex: A regular expression pattern
@param result: The value to be returned if the regular expression matches
@param type: The type of regular expression rule
#TODO better doc
math: return match object groups applied to math function
search: return re.search
match: return re.match
"""
#escape regex as needed
self.escape = escape
if escape:
'''# captures '#' or '#\d+' or '\d+' '''
self.regex = re_escape(regex).replace('#', '(\d+|#\d+|#)')
else:
self.regex = regex
#do an exact match
if type == 'search':
self.re_func = re.search
self.delete_transform_regex = self.regex
elif type == 'match':
self.re_func = re.match
self.delete_transform_regex = ''.join(['^', self.regex])
elif type == 'exact':
self.delete_transform_regex = ''.join(['^', self.regex, '$'])
self.regex = ''.join([self.regex, '$'])
self.re_func = re.match
elif type == 'end':
self.delete_transform_regex = ''.join([self.regex, '$'])
self.regex = ''.join([self.regex, '$'])
self.re_func = re.search
elif type == 'findall':
#TODO this is currently broken, this is not an exact match this is a match to the end of the string
self.delete_transform_regex = self.regex
self.regex = self.regex
self.re_func = re.findall
elif type == 'tilde':
#TODO this is currently broken, this is not an exact match this is a match to the end of the string
self.delete_transform_regex = ''.join(['(?:^|~)', self.regex, '(?:~|$)'])
self.regex = ''.join(['(?:^|~)', self.regex, '[ ]*(?:~|$)'])
self.re_func = re.search
elif type == 'tilde-match':
self.delete_transform_regex = ''.join([self.regex, '[ ]*(?:~|$)'])
self.regex = ''.join([self.regex, '[ ]*(?:~|$)'])
self.re_func = re.match
elif type == 'tilde-begin':
self.delete_transform_regex = ''.join(['(?:~|^)', self.regex])
self.regex = ''.join(['(?:~|^)', self.regex])
self.re_func = re.search
self.regex = re.compile(self.regex, re.I) #actual regular expression string
self.delete_transform_regex = re.compile(self.delete_transform_regex, re.I) #actual regular expression string
self.return_type = 'result' #this rule will return a value
self.result = self._validate_result(result)
self.reg = regex
self.type = type
self.match = None #object to hold match object
self.priority = priority
#TODO change this to match RegexMathRule.handle()
[docs] def handle(self, value):
"""Returns True if rule can handle the value. Convert will only be called
if handle returns True."""
if value in [None, '']:
raise ConvertError
#TODO this is not using compiled regular expression
#should be self.regex.match
self.match = self.re_func(self.regex, value)
if self.match:
self.groups = self.match.groups()
#will be the match object or None
return self.match
[docs] def convert(self, **kw):
"""Return result."""
if kw.get('split'):
return [x for x in self.result.split(kw['split'])]
else:
return self.result
def _validate_result(self, result):
"""Helper function to validate the result string. This method can be overridden to do
clean, formating, validation of the result in classes that inherit from RegexRule."""
return result
_formula_func_map = {
'max': (re.compile('max\(|\)$'), max),
'min': (re.compile('min\(|\)$'), min)
}
def _get_result(result, groups, type=None):
'''result = the result we want to calculate for a output variable.
groups = the groups object from the match object on the regex see handle below
type = The type of function we want to calculate
'''
if type in _formula_func_map:
_ff = _formula_func_map[type]
result = _ff[0].sub('',result)
_values = result.split(',')
return _ff[1]([calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in _values])
else:
return calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), result))
[docs]class RegexMathRule(RegexRule):
def __init__(self, regex, result, type, priority, escape=True, **kw):
RegexRule.__init__(self, regex, result, type, priority, escape, **kw)
self.result = self.result.lower()
self.has_result_formula_func = True if 'max' in self.result or 'min' in self.result else False
[docs] def handle(self, value):
RegexRule.handle(self, value)
self.match = self.re_func(self.regex, value)
if self.match:
self.groups = self.match.groups()
return True
else:
return False
[docs] def convert(self, **kw):
"""
For math regex apply groups extracted from value to arthimetic function within self.result
Math:
>>> r = RegexMathRule('Sent to #Day #Y', '#1 + #2(365.25)', 'math')
>>> r.handle('Sent to 10Day 2Y')
True
>>> r.convert()
740.5
"""
#groups are base 0, research tema has been entering there # objects as base one
#subtract 1 from every #\d+ match and return the matched group
#example)
# regex = Sent to #Day #Y
# result = #1 + #2(365.25)
# value = 'Sent to 10Day 2Y'
#return calc('10 + 2(365.25)')
#if a transform result function is present then substitute the variable name in the results
#with the current value in the row
result = self.result
if kw.get('transform_results'):
for var in kw.get('transform_results'):
if var in result:
result = result.replace(var, str(kw['row'][var]))
try:
if kw.get('split'):
'''@todo: Make result a class so we do not need to check for max, min on every result value'''
result = [r if not is_none(r) else 'None' for r in result.split(kw['split'])]
if self.has_result_formula_func:
_out_values = []
for r in result:
if r.startswith('max'):
_out_values.append(_get_result(r, self.groups, 'max'))
elif r.startswith('min'):
_out_values.append(_get_result(r, self.groups, 'min'))
else:
_out_values.append(_get_result(r, self.groups))
return _out_values
else:
#same as above but condensed without evaluation
return [calc(re.sub('(#)(\d+)', lambda c: self.groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in result]
else:
if self.has_result_formula_func:
if result.startswith('max'):
return _get_result(result, self.groups, 'max')
elif result.startswith('min'):
return _get_result(result, self.groups, 'min')
else:
return _get_result(result, self.groups)
else:
return calc(re.sub('(#)(\d+)', lambda c: self.groups[int(c.group(2)) - 1].replace('#', ''), result))
except CalcError:
raise RuleError("CalcError: Invalid Result Formula: %s . Priority = %s" % (self.result, self.priority))
except IndexError:
raise RuleError("Invalid output value: key:%s | value:%s, number index out of range" % (self.regex.pattern, self.result))
except SyntaxError:
raise RuleError("Syntax Error: check rule syntax: key:%s | value:%s" % (self.regex.pattern,self.result))
except AttributeError:
raise RuleError("Attribute Error: check rule syntax: key:%s | value:%s" % (self.regex.pattern,self.result))
except NameError:
raise RuleError("NameError: make sure the variable you listed is in the ruleset.variables metadata: key:%s | result:%s | %s" % (self.regex.pattern,self.result, kw.get(['row'], {'id':''})['id']))
def _validate_result(self, result):
"""Clean the result formulas"""
#replace [ ] with ( )
result = result.replace('[', '(')
result = result.replace(']', ')')
result = result.replace('{', '(')
result = result.replace('}', ')')
result = result.replace(' ', '')
return result
class RegexMathFindallRule(RegexMathRule):
def handle(self, value):
if value in [None, '']:
raise ConvertError
self.groups = self.re_func(self.regex, value)
self.match = self.regex.search(value) #need match object for delete transform rule
return self.match #return None or match object
def convert(self, **kw):
result = self.result
if kw.get('transform_results'):
for var in kw.get('transform_results'):
if var in result:
result = result.replace(var, str(kw['row'][var]))
try:
out = []
if kw.get('split'):
'''@todo: Make result a class so we do not need to check for max, min on every result value'''
result = [r if not is_none(r) else 'None' for r in result.split(kw['split'])]
#findall has one or many groups we need to select the highest
for groups in self.groups:
if not isinstance(groups, tuple): groups = tuple([groups])
out.append([calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in result])
#determine which result formula we want to evaluate max on
c = 0
for r in result:
if '#' in r: #if multiple formulas use first
eval_index = c
break
else:
c+=1
#return the largest item
return sorted(out, key = lambda k: k[eval_index])[-1]
else:
for groups in self.groups:
out.append(calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), result)))
return sorted(out)[-1]
except CalcError:
raise RuleError("CalcError: Invalid Result Formula: %s . Priority = %s" % (self.result, self.priority))
except IndexError:
raise RuleError("Invalid output value: key:%s | value:%s, number index out of range. Priority = %s" % (self.regex.pattern, self.result, self.priority))
except SyntaxError:
raise RuleError("Syntax Error: check rule syntax: key:%s | value:%s. Priority = %s" % (self.regex.pattern,self.result, self.priority))
except AttributeError:
raise RuleError("Attribute Error: check rule syntax: key:%s | value:%s. Priority = %s" % (self.regex.pattern,self.result, self.priority))
except NameError:
raise RuleError("NameError: make sure the variable you listed is in the ruleset.variables metadata: key:%s | result:%s | %s. Priority = %s" % (self.regex.pattern,self.result, kw.get(['row'], {'id':''})['id'], self.priority))
class RegexReturnRule(RegexRule):
"""Return the Match Object of a match regular expression."""
def __init__(self, regex, result, type, priority, escape=True):
RegexRule.__init__(self, regex, result, type, priority, escape)
self.return_type = 'match'
def handle(self, value):
RegexRule.handle(self, value)
self.result = self.re_func(self.regex, value)
return True if self.result else False
class Lookup(object):
def __init__(self, db, tables, log):
self.values = []
self.log = log
self.lookup_tables = tables.split('|')
# for table in tables.split('|'):
# ds = cs.load(source=db['name'], format='mysql', dataset=table, user=db['user'], pwd=db['pass'])
# self.values.append(([row['id'] for row in ds], table))
def validate(self, initial, value, filename):
#iterate over all values and lookups checking to see if each value
#in corresponding lookup list
for value, table in izip(value.split('|'), self.lookup_tables):
try:
if value not in ('', 'None', ':none:') and int(value) not in LOOKUPS[table]:
self.log.write('ref_integ_error', [value, table, LOOKUPS[table], filename, 'Value not present in lookup'], 90)
return False
except:
self.log.write('ref_integ_error', [value, table, LOOKUPS[table], filename, 'Value not present in lookup'], 90)
raise RuleError("Rule validation failed for. Check log.")
return True
# temp_value = value.split('|')
# for v, l in izip(temp_value, self.values):
# try:
# if v not in ('', 'None', ':none:') and int(v) not in l[0]:
# #TODO add validate_rules
# #if segment.validate_rules:
# self.log.write('ref_integ_error', [v, l[1], l[0], filename, 'Value not present in lookup'], 90)
# return False
# except:
# self.log.write('ref_integ_error', [v, l[1], l[0], filename, 'Value not present in lookup'], 90)
# raise RuleError("Rule validation failed for. Check log.")
#
# return True
[docs]class Crosswalk(object):
"""A Crosswalk object is used to do variable conversions either directly
or by accessing the object in the func.py file for a CCHR variable.
The Crosswalk object first loads the crosswalk.txt file specified by the
**filename** parameter. Each line of this file is turned into a rule object,
see _rules for more information. When crosswalk.convert(value) is called
we iterate over the rules list checking to see if a rule can handle the
value. If a rule can handle the value the rules result is returned,
otherwise we raise a ConvertError. An additional **lookup** parameter can
be defined to provide a list of valid return values.
The column delimiter is a tab and the row delimiter is a newline.
So each line in the crosswalk looks like:
**key**\\t**value**\\n
example)
100\tmurder
parameters:
filename - Path to file containing crosswalk information.
lookup - A list of valid values that can be returned by the convert function.
"""
def __init__(self, filename, db_info, segment, log, **kw):
self._re_pattern = re.compile(':re:.+?:')
self.db_info = db_info
#rules
self.rules = defaultdict(dict)
self.reg_rules = []
self.lookup = None
self.log = log
#metadata
self.format = 'string'
self.missing = None
self.unknown = None
self._has_all = False
self._key_rule_map = {}
self.remove_key = False
self.transform_results = False
self.filename = filename
self.vars = []
self.key = {}
self.skip_missing = []
self.skip_unknown = []
self.null_predicate = []
self.key_predicate = None
self.overwrite = []
self.force_unknown = False
self.rule_generator_map = {
':re:search:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'search', self.line_priority),
':re:match:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'match', self.line_priority),
':re:exact:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'exact', self.line_priority),
':re:end:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'end', self.line_priority),
':re:tilde:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde', self.line_priority),
':re:tilde-match:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority),
':re:tilde-begin:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority),
':re:math-s:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'search', self.line_priority),
':re:math-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern, self.line_value, 'match', self.line_priority),
':re:math-e:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'exact', self.line_priority),
':re:math-end:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'end', self.line_priority),
':re:math-f:': lambda args: self._generate_re_rule(RegexMathFindallRule, self.re_pattern,self.line_value, 'findall', self.line_priority),
':re:math-t:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde', self.line_priority),
':re:math-t-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority),
':re:math-t-b:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority),
':re:regex-s:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'search', self.line_priority, False),
':re:regex-m:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'match', self.line_priority, False),
':re:regex-e:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'exact', self.line_priority, False),
':re:regex-end:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'end', self.line_priority, False),
':re:regex-t:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde', self.line_priority, False),
':re:regex-t-m:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority, False),
':re:regex-t-b:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority, False),
':re:regex-math-s:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'search', self.line_priority, False),
':re:regex-math-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'match', self.line_priority, False),
':re:regex-math-e:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'exact', self.line_priority, False),
':re:regex-math-end:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'end', self.line_priority, False),
':re:regex-math-f:': lambda args: self._generate_re_rule(RegexMathFindallRule, self.re_pattern,self.line_value, 'findall', self.line_priority, False),
':re:regex-math-t:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde', self.line_priority, False),
':re:regex-math-t-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority, False),
':re:regex-math-t-b:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority, False),
':re:return-m:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'match', None, False),
':re:return-s:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'search', None, False),
':re:return-e:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'exact', None,False),
':all:': lambda args: self._create_all_rule(),
':key_predicate:': lambda args: self._set_key_predicate(),
':value_predicate:': lambda args: self._set_value_predicate(args[0]),
':overwrite:': lambda args: self._set_overwrite(self.line_value),
':null_predicate:': lambda args: self._set_null_predicate(args[0], args[1]),
':missing:': lambda args: self._set_missing(self.line_value),
':unknown:': lambda args: self._set_unknown(self.line_value),
':lookup:': lambda args: self._set_lookup(args[0], args[1]),
':vars:': lambda args: self._set_vars(args[0], args[1]),
':keys:': lambda args: self._set_keys(args[0], args[1]),
':skip_missing:': lambda args: self._set_skip_missing(args[0]),
':skip_unknown:': lambda args: self._set_skip_unknown(args[0]),
':remove_key:': lambda args: self._set_remove_key(self.line_value),
':transform_results:': lambda args: self._set_transform_results(self.line_value),
':key:': lambda args: self._set_key_rule(args[0]),
':force_unknown:': lambda args: self._set_force_unknown(self.line_value),
}
#load the rules
self.load(filename, db_info, segment, kw.get('prediction', False))
#create a VariableSet of the output variables for the crosswalk
out_vars = cs.variables.VariableSet([(var[0], var[1]) for var in self.vars])
#create a sorted list of KeyMetadata objects
key_meta = [k_m for k_m in sorted([k for k in self.key.itervalues()], key= lambda k:k.order)]
opts = dict(missing=self.missing, unknown=self.unknown, force_unknown=self.force_unknown)
#create the RuleSetMetadata
self.metadata = cs.transform.RuleSetMetadata(key_meta, out_vars, **opts)
self.validate()
def _create_all_rule(self):
self.rules[self.line_priority][self.line_initial] = AllRule(self.line_value)
self._has_all = True
def _generate_re_rule(self, rule_class, pattern, return_value, search_method, priority, escape=True):
if priority is not None:
self.reg_rules.append(rule_class(pattern, return_value, search_method, priority, escape=escape))
else:
self.reg_rules.append(rule_class(pattern, return_value, search_method, escape=escape))
#metadata setters
def _set_lookup(self, line, segment):
self.lookup = Lookup(self.db_info, line.split('\t')[1], self.log)
def _set_skip_missing(self, line):
self.skip_missing = ([k for k in line.split('\t')[1].split('|')])
def _set_force_unknown(self, line_value):
self.force_unknown = str_to_boolean(line_value)
def _set_skip_unknown(self, line):
self.skip_unknown = ([k for k in line.split('\t')[1].split('|')])
def _set_key_predicate(self):
#self.key_predicate = self.line_value
self.key_predicate = re.compile(self.line_value)
def _set_value_predicate(self, line):
if '|' in value:
_vars = [v for v in line[1].split('|')]
values = [v for v in line[2].split('|')]
self.value_predicate = izip(vars, values)
else:
self.value_predicate = (line[1], line[2] if line[2] != ':none:' else None)
def _set_overwrite(self, overwrite):
if '|' in overwrite:
self.overwrite = [o for o in overwrite.split('|')]
else:
self.overwrite.append(overwrite)
def _set_missing(self, missing):
self.missing = [m for m in missing.split('|')] if '|' in missing else missing
def _set_unknown(self, unknown):
self.unknown = [u for u in unknown.split('|')] if '|' in unknown else unknown
def _set_null_predicate(self, line, segment):
#all values must be None
#TODO this should be combined with value_predicate
self.null_predicate = line.split('\t')[1].split('|')
self._validate_vars(self.null_predicate, segment)
def _set_vars(self, line, segment):
vars = line.split('\t')[1].split('|')
try:
for v in vars:
v_info = segment.var_dict[v]
self.vars.append((v, v_info[0], v_info[2], v_info[3]))
except KeyError as e:
raise RuleError('Invalid variable for crosswalk: %s. %s' % (vars, e))
def _set_keys(self, line, segment):
#TODO make this python-etl variables object by looking at metadata
keys = line.split('\t')[1].split('|')
try:
for i, k in enumerate(keys):
v_info = segment.var_dict[k]
key_var = cs.transform.metadata.KeyMetadata(cs.variables.Variable(k, v_info[0]), i, as_str=True, inner_trim=True)
self.key[k] = key_var
#self.key[k] = dict(format=v_info[0], length=v_info[2], precision=v_info[3], order=_c)
except KeyError as e:
raise RuleError('Invalid variable for crosswalk: %s. %s' % (keys, e))
def _set_remove_key(self, r):
self.remove_key = str_to_boolean(r)
def _set_transform_results(self, transform_results):
self.transform_results = [k for k in transform_results.split('|')]
def _set_key_rule(self, line):
rule_info = line.split('\t')
#need to store in map so we can keep track of multiple rules within one crosswalk
self._key_rule_map[rule_info[1]] = rule_info
#rule to replace the key
#:key: cchglitx :replace: 0 o
if re.search('replace', rule_info[2]):
def _replace(key, var_name):
_rule_info = self._key_rule_map[var_name]
return key.replace(_rule_info[3], _rule_info[4] if _rule_info[4] != ':none:' else '')
try:
self.key[rule_info[1]].transform = _replace
except KeyError as e:
raise RuleError('KeyError %s: invalid variable in %s' % (e, self.filename))
elif re.search('sub', rule_info[2]):
#compile the regex
self._key_rule_map[rule_info[1]][3] = re.compile(rule_info[3])
def _sub(key, var_name):
_rule_info = self._key_rule_map[var_name]
return _rule_info[3].sub(_rule_info[4] if _rule_info[4] != ':none:' else '', key)
try:
self.key[rule_info[1]].transform = _sub
except KeyError as e:
raise RuleError('KeyError %s: invalid variable in %s' % (e, self.filename))
elif re.search('predicate_re_replace', rule_info[2]):
#search for the predicate regular expression on key
#if a match is found do a replace on the match.group() of the second regex with the third input string
self._key_rule_map[rule_info[1]][3] = re.compile(rule_info[3])
self._key_rule_map[rule_info[1]][4] = re.compile(rule_info[4])
self._key_rule_map[rule_info[1]][5] = rule_info[5] if rule_info[5] != ':none:' else ''
def _predicate_re_transform(key, var_name):
_rule_info = self._key_rule_map[var_name]
if _rule_info[3].search(key):
replace = _rule_info[4].search(key)
return key.replace(replace.group(), _rule_info[5])
self.key[rule_info[1]].transform = _predicate_re_transform
else:
raise RuleError('Invalid task for crosswalk: %s' % rule_info[2])
[docs] def load(self, filename, db_info, segment, no_space=False):
#for line in codecs.open(filename, 'r', 'utf-8'):
for line in codecs.open(filename, 'r'):
line = line.strip()
split = line.split('\t')
self.line_initial, self.line_value, self.line_priority = None, None, None
if line:
try:
self.line_initial, self.line_value = split[:2]
if no_space:
self.line_initial = self.line_initial.replace(' ', '')
if self.line_initial.startswith(':re:'):
self.re_pattern = self._re_pattern.sub('', self.line_initial)
self.line_initial = self._re_pattern.match(self.line_initial).group()
except ValueError:
print line
print self.filename
raise
if len(split) == 3:
self.line_priority = int(split[2])
else:
self.line_priority = 1
try:
if self.lookup is not None:
if not self.lookup.validate(self.line_initial, self.line_value, filename):
continue
if self.line_initial in self.rule_generator_map:
self.rule_generator_map.get(self.line_initial)((line, segment))
else:
self.rules[self.line_priority][self.line_initial] = (Rule(self.line_initial.lower(), self.line_value))
except sre_constants.error as e:
print "Skipping regular expression rule. Check regex_init log for more details."
self.log.write('regex_init', [e, filename, self.line_initial, self.line_value, self.line_priority], 91, no_add_info=True)
def _validate_vars(self, vars, segment):
for v in vars:
try:
assert(segment.var_dict.get(v))
except AssertionError:
raise RuleError('Invalid variable for crosswalk: %s' % v)
def _validate_regex_rules(self):
patterns = dict()
#TODO MongoDB for log reports
for regex_rule in sorted(self.reg_rules, key=lambda rule: rule.priority, reverse=True):
#validate result matches variables
if len(self.metadata.out_vars) != len(regex_rule.result.split('|')) and 'remove' not in self.filename:
self.log.write('regex_init', ['error:result does not match out_variables', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority, self.metadata.out_vars], 91, no_add_info=True)
#math rules validate the same number of parans
if isinstance(regex_rule, RegexMathRule):
if len(re.findall('\(', regex_rule.result)) != len(re.findall('\)', regex_rule.result)):
self.log.write('regex_init', ['error:Syntax Error, unbalanced parentheses', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)
if re.search('#[ ]*(?!\d+)', regex_rule.result):
self.log.write('regex_init', ['error:Syntax Error, repeating digit group (#) not followed by number reference', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)
_digit_group = [int(x) for x in re.findall('#[ ]*(\d+)', regex_rule.result)] or [0]
if int(max(_digit_group)) > len(re.findall('%s|%s' % (re.escape('(\d+|#\d+|#)'), re.escape('(\d+)')), regex_rule.regex.pattern)):
self.log.write('regex_init', ['error:Syntax Error, formula number index out of range', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)
#validate that the result does not have 4# instead of #4
if re.search('\d#', regex_rule.result):
self.log.write('regex_init', ['potential error:Syntax Error, formula number index before number', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)
#validate rule logic
if regex_rule.type in ('exact', 'tilde') and regex_rule.escape:
continue
for pattern, regex in patterns.iteritems():
if regex_rule.regex.pattern in pattern:
if regex[0] != regex_rule.result.replace(' ', ''):
err = '%serror:inconsistent result' % ('potential ' if regex_rule.type in ('exact', 'tilde', 'match') else '')
self.log.write('regex_init', [err, self.filename, regex_rule.regex.pattern, regex_rule.result.replace(' ', ''), regex_rule.priority, pattern, regex[0], regex[1]], 91, no_add_info=True)
else:
self.log.write('regex_init', ['warning:redundent rule', self.filename, regex_rule.regex.pattern, regex_rule.result.replace(' ', ''), regex_rule.priority, pattern, regex[0], regex[1]], 91, no_add_info=True)
patterns[regex_rule.regex.pattern] = (regex_rule.result.replace(' ', ''), regex_rule.priority)
[docs] def validate(self):
'''Validate the ruleset.'''
if config.debug_validate_regex_rules:
self._validate_regex_rules()
def _convert(self, rule, value, **kw):
"""Helper function for convert"""
if rule.type == 'all':
kw['value'] = value
out_value = rule.convert(**kw)
out = []
if 'split' in kw:
for val, var in izip(out_value, self.vars):
if var[1] in FORMAT_DICT:
if is_none(val):
o = None
else:
if var[1] == 'float':
try:
#hacky fix to deal with precision without transforming to a decimal (I think this is faster)
o = round(round(FORMAT_DICT[var[1]](val), var[3]+1), var[3])
except ValueError:
raise ConvertError("Unable to convert. Check rule regex: %s" % rule.regex.pattern)
else:
o = FORMAT_DICT[var[1]](val)
out.append(o)
else:
raise ConvertError('Unable to convert to %s' % self.format)
return out
if rule.return_type == 'match':
return out_value
if self.vars[0][1] in FORMAT_DICT:
if is_none(out_value):
return None
else:
#convert to correct format
if self.vars[0][1] == 'string':
return out_value
elif self.vars[0][1] == 'float':
#hacky fix to deal with precision without transforming to a decimal (I think this is faster)
return round(round(FORMAT_DICT[self.vars[0][1]](out_value), self.vars[0][3]+1), self.vars[0][3])
else:
return FORMAT_DICT[self.vars[0][1]](out_value)
else:
raise ConvertTypeError('Unable to convert to %s' % self.format)
[docs] def convert(self, key, **kw):
"""
Returns a converted value based on key or raises a ConvertError
if key can not be converted.
Looks to see if there is a rule that can handle a given key.
Split keyword returns the converted result as a list of values,
using the value of the split keyword as the delimiter for the result
list.
"""
#check to see if :all: rule is present
if not self._has_all:
str_key = None if key is None else str(key)
else:
str_key = ':all:'
for rule_group in self.rules.itervalues():
if str_key in rule_group:
return self._convert(rule_group[str_key], str_key, **kw)
if self.reg_rules: #if key not found in rules dict then see if regex rule can handle the key
for rule in sorted(self.reg_rules, key=lambda rule: rule.priority):
if rule.handle(str_key):
if kw.get('remove'):
return (self._convert(rule, str_key, **kw), rule.match.group())
else:
return self._convert(rule, str_key, **kw)
raise ConvertError('Unable to convert to %s' % key)